{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bidirectional LSTM Sentiment Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we use a *bidirectional* LSTM to classify IMDB movie reviews by their sentiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/the-deep-learners/deep-learning-illustrated/blob/master/notebooks/bi_lstm_sentiment_classifier.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import imdb\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, LSTM\n",
    "from keras.layers.wrappers import Bidirectional # new! \n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "from sklearn.metrics import roc_auc_score \n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# output directory name:\n",
    "output_dir = 'model_output/biLSTM'\n",
    "\n",
    "# training:\n",
    "epochs = 6\n",
    "batch_size = 128\n",
    "\n",
    "# vector-space embedding: \n",
    "n_dim = 64 \n",
    "n_unique_words = 10000 \n",
    "max_review_length = 200 # doubled!\n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2 \n",
    "\n",
    "# LSTM layer architecture:\n",
    "n_lstm = 256 \n",
    "drop_lstm = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n",
    "model.add(SpatialDropout1D(drop_embed))\n",
    "model.add(Bidirectional(LSTM(n_lstm, dropout=drop_lstm)))\n",
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 200, 64)           640000    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_1 (Spatial (None, 200, 64)           0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 512)               657408    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 513       \n",
      "=================================================================\n",
      "Total params: 1,297,921\n",
      "Trainable params: 1,297,921\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# LSTM layer parameters double due to both reading directions\n",
    "model.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/6\n",
      "25000/25000 [==============================] - 157s 6ms/step - loss: 0.5720 - acc: 0.6936 - val_loss: 0.3837 - val_acc: 0.8401\n",
      "Epoch 2/6\n",
      "25000/25000 [==============================] - 156s 6ms/step - loss: 0.3675 - acc: 0.8438 - val_loss: 0.3456 - val_acc: 0.8535\n",
      "Epoch 3/6\n",
      "25000/25000 [==============================] - 155s 6ms/step - loss: 0.2764 - acc: 0.8923 - val_loss: 0.3584 - val_acc: 0.8448\n",
      "Epoch 4/6\n",
      "25000/25000 [==============================] - 155s 6ms/step - loss: 0.2223 - acc: 0.9166 - val_loss: 0.3308 - val_acc: 0.8604\n",
      "Epoch 5/6\n",
      "25000/25000 [==============================] - 155s 6ms/step - loss: 0.1814 - acc: 0.9333 - val_loss: 0.3846 - val_acc: 0.8546\n",
      "Epoch 6/6\n",
      "25000/25000 [==============================] - 155s 6ms/step - loss: 0.1526 - acc: 0.9450 - val_loss: 0.3855 - val_acc: 0.8649\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f57b978f358>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# - with this toy dataset, the complex interplay of words over long sentence segments, won't be learned much\n",
    "# - so our CNN picking up location-invariant segments of two to four words that predict review sentiment\n",
    "# - these are simpler and so easier to learn from the data\n",
    "# - CNN therefore outperforms on the IMDB data set\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+\"/weights.04.hdf5\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = model.predict_proba(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFS5JREFUeJzt3X+QXeV93/H3x8jYsWNbAhaGSqLCY9k1zowx3QG5nkkdyxW/Mog/IJWnKQqjqTopTZM00xa3f6gFM4P7i5aZBFcNaoQnMRAaB41NQxUB47ZTMMIQwo8wWgOBrSjaWEJuSk0C+faP+whfxK72rLR718t5v2Z27jnf85x7ngcJfe55ztl7UlVIkvrnXYvdAUnS4jAAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSeWrbYHTiW0047rdasWbPY3ZDe7vvPDF4/+LHF7Yc0jUceeeRPqmpstnY/0gGwZs0a9u7du9jdkN7u9z87eP38A4vZC2laSf64SzungCSppwwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCknuoUAEl+OcmTSZ5I8rUk701ydpKHkuxLckeSk1vb97T1ibZ9zdD7fLHVn0ly4cIMSZLUxay/CZxkJfAPgHOq6v8luRPYBFwC3FRVtyf5CrAFuKW9HqqqjyTZBHwZ+JtJzmn7fQL4S8DvJ/loVb2xICOTpBO05tpvLtqxn7/x0gU/RtcpoGXAjyVZBrwPeAn4HHBX274TuLwtb2zrtO3rk6TVb6+q16rqOWACOP/EhyBJOh6zBkBV/S/gXwMvMPiH/zDwCPBKVb3emk0CK9vySuDFtu/rrf2pw/Vp9nlTkq1J9ibZOzU1dTxjkiR1MGsAJFnB4NP72Qymbt4PXDxN0zqyywzbZqq/tVC1varGq2p8bGzWL7OTJB2nLlNAnweeq6qpqvpz4HeAvwYsb1NCAKuA/W15ElgN0LZ/CDg4XJ9mH0nSiHUJgBeAdUne1+by1wNPAfcDV7Q2m4G72/Kutk7bfl9VVatvancJnQ2sBb49P8OQJM3VrHcBVdVDSe4CvgO8DjwKbAe+Cdye5Eutdmvb5Vbgq0kmGHzy39Te58l2B9FT7X2u8Q4gSVo8nR4IU1XbgG1HlZ9lmrt4quoHwJUzvM8NwA1z7KMkaQH4m8CS1FMGgCT1lAEgST1lAEhSTxkAktRTBoAk9ZQBIEk9ZQBIUk8ZAJLUUwaAJPWUASBJPWUASFJPGQCS1FMGgCT1lAEgST1lAEhST3V5KPzHkjw29PP9JL+U5JQku5Psa68rWvskuTnJRJLHk5w39F6bW/t9STbPfFRJ0kKbNQCq6pmqOreqzgX+KvAq8HXgWmBPVa0F9rR1gIsZPO93LbAVuAUgySkMnip2AYMniW07EhqSpNGb6xTQeuC7VfXHwEZgZ6vvBC5vyxuB22rgQWB5kjOBC4HdVXWwqg4Bu4GLTngEkqTjMtcA2AR8rS2fUVUvAbTX01t9JfDi0D6TrTZTXZK0CDoHQJKTgcuA356t6TS1Okb96ONsTbI3yd6pqamu3ZMkzdFczgAuBr5TVS+39Zfb1A7t9UCrTwKrh/ZbBew/Rv0tqmp7VY1X1fjY2NgcuidJmou5BMAX+OH0D8Au4MidPJuBu4fqV7W7gdYBh9sU0b3AhiQr2sXfDa0mSVoEy7o0SvI+4G8Af3eofCNwZ5ItwAvAla1+D3AJMMHgjqGrAarqYJLrgYdbu+uq6uAJj0CSdFw6BUBVvQqcelTtewzuCjq6bQHXzPA+O4Adc++mJGm++ZvAktRTBoAk9VSnKaClas2131yU4z5/46WLclxJmgvPACSppwwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCknjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeqpTgGQZHmSu5L8UZKnk3w6ySlJdifZ115XtLZJcnOSiSSPJzlv6H02t/b7kmye+YiSpIXW9Qzg3wO/V1V/Bfgk8DRwLbCnqtYCe9o6wMXA2vazFbgFIMkpwDbgAuB8YNuR0JAkjd6sAZDkg8BPArcCVNWfVdUrwEZgZ2u2E7i8LW8EbquBB4HlSc4ELgR2V9XBqjoE7AYumtfRSJI663IG8GFgCvhPSR5N8utJ3g+cUVUvAbTX01v7lcCLQ/tPttpM9bdIsjXJ3iR7p6am5jwgSVI3XQJgGXAecEtVfQr4v/xwumc6maZWx6i/tVC1varGq2p8bGysQ/ckScejSwBMApNV9VBbv4tBILzcpnZorweG2q8e2n8VsP8YdUnSIpg1AKrqfwMvJvlYK60HngJ2AUfu5NkM3N2WdwFXtbuB1gGH2xTRvcCGJCvaxd8NrSZJWgTLOrb7BeA3k5wMPAtczSA87kyyBXgBuLK1vQe4BJgAXm1tqaqDSa4HHm7trquqg/MyCknSnHUKgKp6DBifZtP6adoWcM0M77MD2DGXDkqSFoa/CSxJPWUASFJPGQCS1FMGgCT1lAEgST1lAEhSTxkAktRTBoAk9ZQBIEk9ZQBIUk8ZAJLUUwaAJPWUASBJPWUASFJPGQCS1FMGgCT1VKcASPJ8kj9M8liSva12SpLdSfa11xWtniQ3J5lI8niS84beZ3Nrvy/J5pmOJ0laeHM5A/ipqjq3qo48GexaYE9VrQX2tHWAi4G17WcrcAsMAgPYBlwAnA9sOxIakqTRO5EpoI3Azra8E7h8qH5bDTwILE9yJnAhsLuqDlbVIWA3cNEJHF+SdAK6BkAB/zXJI0m2ttoZVfUSQHs9vdVXAi8O7TvZajPVJUmLoNND4YHPVNX+JKcDu5P80THaZppaHaP+1p0HAbMV4KyzzurYPUnSXHU6A6iq/e31APB1BnP4L7epHdrrgdZ8Elg9tPsqYP8x6kcfa3tVjVfV+NjY2NxGI0nqbNYASPL+JB84sgxsAJ4AdgFH7uTZDNzdlncBV7W7gdYBh9sU0b3AhiQr2sXfDa0mSVoEXaaAzgC+nuRI+9+qqt9L8jBwZ5ItwAvAla39PcAlwATwKnA1QFUdTHI98HBrd11VHZy3kUiS5mTWAKiqZ4FPTlP/HrB+mnoB18zwXjuAHXPvpiRpvvmbwJLUUwaAJPWUASBJPWUASFJPGQCS1FMGgCT1lAEgST1lAEhSTxkAktRTBoAk9ZQBIEk9ZQBIUk8ZAJLUUwaAJPWUASBJPWUASFJPGQCS1FOdAyDJSUkeTfKNtn52koeS7EtyR5KTW/09bX2ibV8z9B5fbPVnklw434ORJHU3lzOAXwSeHlr/MnBTVa0FDgFbWn0LcKiqPgLc1NqR5BxgE/AJ4CLg15KcdGLdlyQdr04BkGQVcCnw6209wOeAu1qTncDlbXljW6dtX9/abwRur6rXquo5Bg+NP38+BiFJmruuZwD/DvjHwF+09VOBV6rq9bY+CaxsyyuBFwHa9sOt/Zv1afZ5U5KtSfYm2Ts1NTWHoUiS5mLWAEjy08CBqnpkuDxN05pl27H2+WGhantVjVfV+NjY2GzdkyQdp2Ud2nwGuCzJJcB7gQ8yOCNYnmRZ+5S/Ctjf2k8Cq4HJJMuADwEHh+pHDO8jSRqxWc8AquqLVbWqqtYwuIh7X1X9LeB+4IrWbDNwd1ve1dZp2++rqmr1Te0uobOBtcC3520kkqQ56XIGMJN/Atye5EvAo8CtrX4r8NUkEww++W8CqKonk9wJPAW8DlxTVW+cwPElSSdgTgFQVQ8AD7TlZ5nmLp6q+gFw5Qz73wDcMNdOSpLmn78JLEk9ZQBIUk8ZAJLUUwaAJPWUASBJPWUASFJPGQCS1FMGgCT1lAEgST1lAEhSTxkAktRTBoAk9ZQBIEk9ZQBIUk8ZAJLUUwaAJPVUl4fCvzfJt5P8QZInk/yLVj87yUNJ9iW5I8nJrf6etj7Rtq8Zeq8vtvozSS5cqEFJkmbX5QzgNeBzVfVJ4FzgoiTrgC8DN1XVWuAQsKW13wIcqqqPADe1diQ5h8HjIT8BXAT8WpKT5nMwkqTuujwUvqrqT9vqu9tPAZ8D7mr1ncDlbXljW6dtX58krX57Vb1WVc8BE0zzSElJ0mh0ugaQ5KQkjwEHgN3Ad4FXqur11mQSWNmWVwIvArTth4FTh+vT7CNJGrFOAVBVb1TVucAqBp/aPz5ds/aaGbbNVH+LJFuT7E2yd2pqqkv3JEnHYU53AVXVK8ADwDpgeZJlbdMqYH9bngRWA7TtHwIODten2Wf4GNuraryqxsfGxubSPUnSHHS5C2gsyfK2/GPA54GngfuBK1qzzcDdbXlXW6dtv6+qqtU3tbuEzgbWAt+er4FIkuZm2exNOBPY2e7YeRdwZ1V9I8lTwO1JvgQ8Ctza2t8KfDXJBINP/psAqurJJHcCTwGvA9dU1RvzOxxJUlezBkBVPQ58apr6s0xzF09V/QC4cob3ugG4Ye7dlCTNN38TWJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCknjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeqpLk8E0xytufabi3Lc52+8dFGOK2lp6vJM4NVJ7k/ydJInk/xiq5+SZHeSfe11Rasnyc1JJpI8nuS8offa3NrvS7J5pmNKkhZelymg14FfqaqPA+uAa5KcA1wL7KmqtcCetg5wMYMHvq8FtgK3wCAwgG3ABQweJbntSGhIkkZv1gCoqpeq6jtt+f8ATwMrgY3AztZsJ3B5W94I3FYDDwLLk5wJXAjsrqqDVXUI2A1cNK+jkSR1NqeLwEnWMHhA/EPAGVX1EgxCAji9NVsJvDi022SrzVSXJC2CzheBk/w48J+BX6qq7yeZsek0tTpG/ejjbGUwdcRZZ53VtXuS3sEW68aKd7pOZwBJ3s3gH//frKrfaeWX29QO7fVAq08Cq4d2XwXsP0b9Lapqe1WNV9X42NjYXMYiSZqDLncBBbgVeLqq/u3Qpl3AkTt5NgN3D9WvancDrQMOtymie4ENSVa0i78bWk2StAi6TAF9BvjbwB8meazV/ilwI3Bnki3AC8CVbds9wCXABPAqcDVAVR1Mcj3wcGt3XVUdnJdRSJLmbNYAqKr/zvTz9wDrp2lfwDUzvNcOYMdcOihJWhh+FYQk9ZQBIEk9ZQBIUk8ZAJLUUwaAJPWUASBJPWUASFJPGQCS1FMGgCT1lAEgST1lAEhSTxkAktRTBoAk9ZQBIEk91fmRkPrRt5iPzXv+xksX7diSjo9nAJLUU54BSOrMh7O/s3R5JvCOJAeSPDFUOyXJ7iT72uuKVk+Sm5NMJHk8yXlD+2xu7fcl2TzdsSRJo9NlCug3gIuOql0L7KmqtcCetg5wMbC2/WwFboFBYADbgAuA84FtR0JDkrQ4Zg2AqvoWcPTD2zcCO9vyTuDyofptNfAgsDzJmcCFwO6qOlhVh4DdvD1UJEkjdLwXgc+oqpcA2uvprb4SeHGo3WSrzVR/myRbk+xNsndqauo4uydJms18XwTONLU6Rv3txartwHaA8fHxadvoR89iXRz09lPp+B1vALyc5MyqeqlN8Rxo9Ulg9VC7VcD+Vv/sUfUHjvPYUq95J47my/FOAe0CjtzJsxm4e6h+VbsbaB1wuE0R3QtsSLKiXfzd0GqSpEUy6xlAkq8x+PR+WpJJBnfz3AjcmWQL8AJwZWt+D3AJMAG8ClwNUFUHk1wPPNzaXVdVR19YluZssT4N3/7h7wGwyU/jWsJmDYCq+sIMm9ZP07aAa2Z4nx3Ajjn1TpK0YPwqCEnqKQNAknrKAJCknjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCknjIAJKmnRh4ASS5K8kySiSTXjvr4kqSBkQZAkpOAXwUuBs4BvpDknFH2QZI0MOozgPOBiap6tqr+DLgd2DjiPkiSGH0ArAReHFqfbDVJ0ojN+lD4eZZpavWWBslWYGtb/dMkzxznsU4D/uQ4912qHPOIfPrNpZ8e9aHBP+deyJdPaMx/uUujUQfAJLB6aH0VsH+4QVVtB7af6IGS7K2q8RN9n6XEMfeDY+6HUYx51FNADwNrk5yd5GRgE7BrxH2QJDHiM4Cqej3J3wfuBU4CdlTVk6PsgyRpYNRTQFTVPcA9IzjUCU8jLUGOuR8ccz8s+JhTVbO3kiS94/hVEJLUU0s+AGb7aokk70lyR9v+UJI1o+/l/Oow5n+Y5KkkjyfZk6TTLWE/yrp+hUiSK5JUkiV/x0iXMSf5mfZn/WSS3xp1H+dbh7/bZyW5P8mj7e/3JYvRz/mSZEeSA0memGF7ktzc/ns8nuS8ee1AVS3ZHwYXkr8LfBg4GfgD4Jyj2vw94CtteRNwx2L3ewRj/ingfW355/sw5tbuA8C3gAeB8cXu9wj+nNcCjwIr2vrpi93vEYx5O/Dzbfkc4PnF7vcJjvkngfOAJ2bYfgnwXxj8DtU64KH5PP5SPwPo8tUSG4GdbfkuYH2S6X4hbamYdcxVdX9VvdpWH2Tw+xZLWdevELke+JfAD0bZuQXSZcx/B/jVqjoEUFUHRtzH+dZlzAV8sC1/iKN+j2ipqapvAQeP0WQjcFsNPAgsT3LmfB1/qQdAl6+WeLNNVb0OHAZOHUnvFsZcv05jC4NPEEvZrGNO8ilgdVV9Y5QdW0Bd/pw/Cnw0yf9I8mCSi0bWu4XRZcz/HPjZJJMM7ib8hdF0bdEs6NfnjPw20Hk261dLdGyzlHQeT5KfBcaBv76gPVp4xxxzkncBNwE/N6oOjUCXP+dlDKaBPsvgLO+/JfmJqnplgfu2ULqM+QvAb1TVv0nyaeCrbcx/sfDdWxQL+u/XUj8DmPWrJYbbJFnG4LTxWKdcP+q6jJkknwf+GXBZVb02or4tlNnG/AHgJ4AHkjzPYK501xK/ENz17/bdVfXnVfUc8AyDQFiquox5C3AnQFX9T+C9DL4n6J2q0//vx2upB0CXr5bYBWxuy1cA91W7urJEzTrmNh3yHxj847/U54VhljFX1eGqOq2q1lTVGgbXPS6rqr2L09150eXv9u8yuOBPktMYTAk9O9Jezq8uY34BWA+Q5OMMAmBqpL0crV3AVe1uoHXA4ap6ab7efElPAdUMXy2R5Dpgb1XtAm5lcJo4weCT/6bF6/GJ6zjmfwX8OPDb7Xr3C1V12aJ1+gR1HPM7Sscx3wtsSPIU8Abwj6rqe4vX6xPTccy/AvzHJL/MYCrk55byB7okX2MwhXdau66xDXg3QFV9hcF1jkuACeBV4Op5Pf4S/m8nSToBS30KSJJ0nAwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCknvr/VtRXtJ5fLtQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f234c7c21d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'93.47'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
